package top.yudoge.dogeeditor.llm.util.tokenizer

interface Tokenizer {
    fun tokenize(input: String): List<String>
}